#include"PageLib.h"
#include"tinyxml2.h"
#include <regex>
#include<iostream>
#include<sstream>
#include<fstream>
#include<algorithm>
#include<nlohmann/json.hpp>

using std::cout;
using std::cerr;
using std::ostringstream;
using std::ofstream;
using std::all_of;


PageLib::PageLib(string & path)
    :_dirscanner(path)
{}

int getbytenum_utf8(char byte)
{
    int bytenum = 0;
    for (int i = 0; i < 6; ++i)
    {
        if (byte & (1 << (7 - i)))
        {
            ++bytenum;
        }
        else
            break;
    }
    return bytenum = bytenum == 0 ? 1 : bytenum;
}
    void PageLib::create()
{
    using namespace tinyxml2;
    //创建网页库
    vector<string> filepath=_dirscanner.getpath();
    XMLDocument doc;
    int docid=0;
    size_t begin,end=-1;
    int nums=-1;
            std::regex reg("<[^>]+>");
            std::regex reg1("&[a-zA-Z]+;");
    for(auto& i:filepath)
    {
        doc.LoadFile(i.c_str());
        if(doc.ErrorID())
        {
            cerr<<" load file failed "<<i<<"\n";
        }
        XMLElement *itemNode=doc.FirstChildElement("rss")->FirstChildElement("channel")->FirstChildElement("item");
        while(itemNode)
        {
            string title = itemNode->FirstChildElement("title")->GetText();
            string link = itemNode->FirstChildElement("link")->GetText();
            string description;
            XMLElement* point = itemNode->FirstChildElement("description");
            XMLElement* contenpoint = itemNode->FirstChildElement("content");
            if(contenpoint)
            {
                description=contenpoint->GetText();
                cout<< " have content\n";
            }
            else if(point)
            {
                description=point->GetText();
            }
            else{
                itemNode = itemNode->NextSiblingElement("item");
                continue;
            }
            description=regex_replace(description,reg,"");
            title=regex_replace(title,reg,"");
            link=regex_replace(link,reg,"");
            description=regex_replace(description,reg1,"");
            if(all_of(description.begin(),description.end(),[](char r)
            {
                return getbytenum_utf8(r)!=3;
            })||description.size()<20)
            {
                itemNode = itemNode->NextSiblingElement("item");
                continue;
            }

            ++docid;

        string essay;
        ostringstream oss;
            oss<<"<doc>\n\t<docid>"<<docid<<
                "</docid>\n\t<title>"<<title<<
                "</title>\n\t<link>"<<link<<
                "</link>\n\t<content><![CDATA["<<description<<"]]></content>\n</doc>\n";
            essay=oss.str();
            cout<<essay;
            begin = end+1;
            end += essay.size();

            _offsetLib[docid]={begin,end};

            _pages.push_back(essay);
            itemNode = itemNode->NextSiblingElement("item");
        }
       ++ nums;
        if(nums%500==0)
        {
            store();
            _pages.clear();
        }
    }
    if(nums%500!=0)
    {
            store();
    }
    storeoff();
}

    void PageLib::store()
{
    ofstream ofs("/home/sakura/all_file/search_engine/data/ripage.dat",std::ios::app);
    for(auto& i: _pages)
    {
        ofs<<i;
    }
    ofs.close();
}
    void PageLib::storeoff()
{
    ofstream Offset("/home/sakura/all_file/search_engine/data/offset.dat");
    nlohmann::json json = _offsetLib;
    Offset<<json.dump();
    Offset.close();
}
